<?php
require './vendor/autoload.php';
use phpspider\core\phpspider;
use phpspider\core\selector;
use phpspider\core\db;
use phpspider\core\queue;
// https://doc.phpspider.org/callback.html
/* Do NOT delete this comment */
/* 不要删除这段注释 */

$configs = array(
    'name' => '数据',
    'log_show' =>true,
    'tasknum' => 8,
    'save_running_state'=>true,
    'queue_config' => array(
        'host'      => '127.0.0.1',
        'port'      => 6379,
        'pass'      => '123321',
        'db'        => 5,
        'prefix'    => 'phpspider',
        'timeout'   => 30,
    ),
    //爬取的域名列表  
    'domains' => array(
        'www.stats.gov.cn',
    ), 
    //抓取的起点
    'scan_urls' => array(
        'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2020/index.html',
    ),
    'max_try' => 5,
    'fields' => array(
    ),
);
<<<EOF
CREATE TABLE `address` (
    `id` int(11) NOT NULL AUTO_INCREMENT,
    `name` varchar(50) NOT NULL DEFAULT NULL COMMENT "名称",
    `code` varchar(30) NOT NULL DEFAULT NULL COMMENT "区划代码",
    `pid` int(11) NOT NULL DEFAULT NULL COMMENT "上级id",
    `path` varchar(50) NOT NULL DEFAULT NULL COMMENT "id路径",
    `grade` tinyint(1) NOT NULL DEFAULT NULL COMMENT "层级",
    PRIMARY KEY (`id`)
  ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
EOF;
db::set_connect('default', array(
    'host'  => '127.0.0.1',
    'port'  => 3306,
    'user'  => 'root',
    'pass'  => '123321',
    'name'  => 'demo',
));
db::_init();

queue::set_connect('default', array(
    'host'      => '127.0.0.1',
    'port'      => 6379,
    'pass'      => '123321',
    'db'        => 5,
    'prefix'    => 'shengshi',
    'timeout'   => 30,
));
queue::init();

$spider = new phpspider($configs);
$spider->on_content_page = function($page, $content,$phpspider) 
{
    $ins = [];
    $datas = selector::select($content, "//tr[@class='provincetr']/td");
    if (!empty($datas)) {
        foreach ($datas as $data) {
            preg_match("/<a href=\"(.*?).html\">(.*?)<\/a>/",$data,$matches);
            $name = trim(str_replace('<br/>','',$matches[2]));
            if ($name) {
                $id = db::insert('address',['name'=>$name,'code'=>$matches[1],'pid'=>0,'path'=>'0,','grade'=>1]);
                queue::set($matches[1],json_encode([$id,'0,'.$id.',',$matches[1]]));
            }
        }
    } else {
        $datas = selector::select($content, "//tr[@class='villagetr']/td//text()");
        if (!empty($datas)) {
            $count = count($datas);
            $key = substr($datas[0],0,9);
            $path = queue::get($key);
            queue::del($key);
            $path = !empty($path) ? json_decode($path,true) : [];
            for ($i = 0;$i < $count;$i+=3) {
                if ($datas[$i+2] && $datas[$i] != $path[2]) 
                    $ins[] = ['name'=>$datas[$i+2],'code'=>$datas[$i],'pid'=>$path[0],'path'=>$path[1],'grade'=>5];
            }
            if (!empty($ins)) db::insert_batch('address',$ins);
        } else {
            $datas = selector::select($content, "//tr[@class='citytr']/td//text()");
            $sub1 = 2;
            $sub2 = 4;
            $grade = 2;
            if (empty($datas)) {
                $datas = selector::select($content, "//tr[@class='countytr']/td//text()");
                $sub1 = 4;
                $sub2 = 6;
                $grade = 3;
            }
            if (empty($datas)) {
                $datas = selector::select($content, "//tr[@class='towntr']/td//text()");
                $sub1 = 6;
                $sub2 = 9;
                $grade = 4;
            }
            if (!empty($datas)) {
                $count = count($datas);
                $key = substr($datas[0],0,$sub1);
                $path = queue::get($key);
                queue::del($key);
                $path = !empty($path) ? json_decode($path,true) : [];
                for ($i = 0;$i < $count;$i+=2) {
                    if ($datas[$i+1] && $datas[$i] != $path[2]) {
                        $data = ['name'=>$datas[$i+1],'code'=>substr($datas[$i],0,$sub2),'pid'=>$path[0],'path'=>$path[1],'grade'=>$grade];
                        $id = db::insert('address',$data);
                        queue::set(substr($datas[$i],0,$sub2),json_encode([$id,$path[1].$id.',',$datas[$i]]));
                    }
                }
            }
        }
    }
    return $data;
};
$spider->start();
